﻿/*======================================================================
== Copyright : BlueCurve (c)
== Licence   : Gnu/GPL v2.x
== Author    : Teddy Albina
== Email     : bluecurveteam@gmail.com
== Web site  : http://www.codeplex.com/BlueCurve
========================================================================*/
using System;
using System.Collections;
using System.Text.RegularExpressions;
using System.Collections.Generic;

namespace BlueCurve.Common.Ngrams
{
    public class Tokenizer
    {
        /// <summary>
        /// Ngram partition
        /// </summary>
        public string[] Partition(string input, string culturetag)
        {
            Regex r = new Regex("([ \\t{}():;. \n])");
            string[] tokens = r.Split(input.ToLower(System.Globalization.CultureInfo.GetCultureInfoByIetfLanguageTag(culturetag)));
            List<string> filter = new List<string>();

            for (int i = 0; i < tokens.Length; i++)
            {
                MatchCollection mc = r.Matches(tokens[i]);
                if (mc.Count <= 0 && tokens[i].Trim().Length > 0
                    && !filter.Contains(tokens[i]))
                    filter.Add(tokens[i]);
            }
            return filter.ToArray();;
        }



        public virtual string[] Partition(string input)
        {
            Regex r = new Regex("([ \\t{}():;. \n])");
            string[] tokens = r.Split(input.ToLower(System.Globalization.CultureInfo.InstalledUICulture));
            List<string> filter = new List<string>();

            for (int i = 0; i < tokens.Length; i++)
            {
                MatchCollection mc = r.Matches(tokens[i]);
                if (mc.Count <= 0 && tokens[i].Trim().Length > 0
                    && !filter.Contains(tokens[i]))
                    filter.Add(tokens[i]);
            }
            return filter.ToArray(); ;
        }
    }
}
